/*
 *    DynamicArffLoader.java
 *    @author Manuel Baena (mbaena@uma.es)
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

package moa.core.converters;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.Reader;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import moa.streams.DynamicInstanceStream.StructuralChangeInstance;
import moa.streams.DynamicInstanceStream.StructuralChangeHelper;
import moa.streams.DynamicInstanceStream.StructuralChangeType;
import weka.core.Attribute;
import weka.core.DenseInstance;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.RevisionHandler;
import weka.core.RevisionUtils;
import weka.core.SparseInstance;
import weka.core.Utils;
import weka.core.converters.AbstractFileLoader;
import weka.core.converters.IncrementalConverter;
import weka.core.converters.Loader;

/**
 * Reads a source that is in darff (dynamic attribute relation file format)
 * format.
 * <p/>
 * 
 * @author Manuel Baena (mbaena@uma.es)
 * @version $Revision: 1 $
 * @see Loader
 */
public class DynamicArffLoader extends AbstractFileLoader implements
		IncrementalConverter {

	/** for serialization */
	static final long serialVersionUID = 513415L;

	/** the file extension */
	public static String FILE_EXTENSION = ".darff";
	public static String FILE_EXTENSION_COMPRESSED = FILE_EXTENSION + ".gz";

	/** The reader for the source file. */
	protected transient Reader m_sourceReader = null;

	/** The parser for the DARFF file */
	protected transient DynamicArffReader m_ArffReader = null;

	private List<StructuralChangeInstance> lastStructuralChangesList;

	/**
	 * Reads data from an DARFF file.
	 * <p/>
	 */
	public static class DynamicArffReader implements RevisionHandler {

		private final static String COMMENT_OR_NULL_RE = "^%|^\\s|\\A\\Z";
		private final static String RELATION_RE = "^(@relation)\\s+(\\w+)\\s*$";
		private final static Pattern RELATION_PATTERN = Pattern.compile(
				COMMENT_OR_NULL_RE + "|" + RELATION_RE,
				Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
		
		private final static String NUMERIC_ATT_RE = "(" + Attribute.ARFF_ATTRIBUTE_NUMERIC + "|"
				+ Attribute.ARFF_ATTRIBUTE_REAL + "|"
				+ Attribute.ARFF_ATTRIBUTE_INTEGER + ")";
		private final static String NOMINAL_ATT_RE = "\\{((?:(?:[^\\}\"']*|\"[^\"]*\"|'[^']*'),?)*)\\}";
		private final static String STRING_ATT_RE = "(string)";
		private final static String DATE_FORMAT_RE = "([^\\s\"']+)|\"([^\"]*)\"|'([^']*)'";
		private final static String DATE_ATT_RE = "(date)\\s*(?:" + DATE_FORMAT_RE + ")?";
		private final static String ATTRIBUTE_DEFINITION_RE = "(\\w+)\\s+(?:" + NUMERIC_ATT_RE + "|"
				+ NOMINAL_ATT_RE + "|" + STRING_ATT_RE + "|" + DATE_ATT_RE + ")";
		private final static String ATTRIBUTE_RE = "^(@attribute)\\s+" + ATTRIBUTE_DEFINITION_RE + "\\s*$";
		private final static String LABEL_RE = "^(@label)\\s+(" + NOMINAL_ATT_RE + ")\\s*$";	
		private final static String DATA_RE = "^(@data)\\s*$";
		private final static Pattern ATTRIBUTE_PATTERN = Pattern.compile(COMMENT_OR_NULL_RE + "|"
				+ ATTRIBUTE_RE + "|" + DATA_RE + "|" + LABEL_RE, Pattern.CASE_INSENSITIVE
				| Pattern.DOTALL);
		private final static Pattern ATTRIBUTE_DEF_VALUE_PATTERN = Pattern
				.compile("(?:([^\\s\"',]+)|\"([^\"]*)\"|'([^']*)'),?");	

		private final static String ADD_ATTRIBUTE_RE = "@addattribute\\s+" + ATTRIBUTE_DEFINITION_RE;
		private final static String DEL_ATTRIBUTE_RE = "@delattribute\\s+(\\w+)";
		private final static String ADD_ATTRIBUTE_VALUE_RE = "@addattributevalue\\s+(\\w+)\\s+" + ATTRIBUTE_DEF_VALUE_PATTERN;
		private final static String DEL_ATTRIBUTE_VALUE_RE = "@delattributevalue\\s+(\\w+)\\s+" + ATTRIBUTE_DEF_VALUE_PATTERN;
		private final static String ADD_LABELS_RE = "@addlabels\\s+(" + NOMINAL_ATT_RE + ")"; //TODO add range
		private final static String DEL_LABELS_RE = "@dellabels\\s+(" + NOMINAL_ATT_RE + ")"; //TODO del range
		private final static Pattern STRUCTURAL_PATTERN = Pattern.compile(ADD_ATTRIBUTE_RE + 
								"|" + DEL_ATTRIBUTE_RE +
								"|" + ADD_ATTRIBUTE_VALUE_RE + 
								"|" + DEL_ATTRIBUTE_VALUE_RE + 
								"|" + ADD_LABELS_RE + 
								"|" + DEL_LABELS_RE, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
		
		
		private final static String ATTRIBUTE_VALUE_RE = "(?:(\\?)|(\\d+(?:\\.\\d+)?)|([^\\s\"',]+)|\"([^\"]*)\"|'([^']*)')";

		private final static Pattern ATTRIBUTE_VALUE_PATTERN = Pattern.compile("\\s*" + ATTRIBUTE_VALUE_RE
				+ "\\s*,?");
		
		private final static Pattern ATTRIBUTE_SPARSE_VALUE_PATTERN = Pattern.compile("\\s*(\\d+)\\s+"
				+ ATTRIBUTE_VALUE_RE + "\\s*(?:,|(\\}))");
		
		private final static Pattern WEIGHT_PATTERN = Pattern
				.compile("\\s*(?:\\{\\s*(\\d+(?:\\.\\d+)?)\\s*\\}\\s*$|$)");
		
		

		/** the actual data */
		protected Instances m_Data;

		private LineNumberReader reader;
		private List<StructuralChangeInstance> m_StructuralChangeList;

		/**
		 * Reads the data completely from the reader. The data can be accessed
		 * via the <code>getData()</code> method.
		 * 
		 * @param reader
		 *            the reader to use
		 * @throws IOException
		 *             if something goes wrong
		 * @see #getData()
		 */
		public DynamicArffReader(Reader reader) throws IOException {
			if (reader instanceof LineNumberReader) {
				this.reader = (LineNumberReader) reader;
			} else {
				this.reader = new LineNumberReader(reader);
			}
			this.m_StructuralChangeList = new LinkedList<StructuralChangeInstance>();
			readHeader();
		}

		/**
		 * Throws error message with line number and last token read.
		 * 
		 * @param msg
		 *            the error message to be thrown
		 * @throws IOException
		 *             containing the error message
		 */
		protected void errorMessage(String msg) throws IOException {
			String str = "line " + reader.getLineNumber() + ":" + msg;
			throw new IOException(str);
		}

		/**
		 * Reads a single instance using the tokenizer and returns it.
		 * 
		 * @param structure
		 *            the dataset header information, will get updated in case
		 *            of string or relational attributes
		 * @return null if end of file has been reached
		 * @throws IOException
		 *             if the information is not read successfully
		 */
		public Instance readInstance() throws IOException {

			// Check if any attributes have been declared.
			if (m_Data.numAttributes() == 0) {
				errorMessage("no header information available");
			}

			String line = reader.readLine();
			while (line != null && line.startsWith("@")) {
				addStructuralChange(line);
				line = reader.readLine();
			}

			// Parse instance
			if (line != null && line.startsWith("{")) {
				return getInstanceSparse(line);
			} else if (line != null) {
				return getInstanceFull(line);
			} else {
				return null;
			}
		}

		private void addStructuralChange(String line) throws IOException {
			Matcher matcher = STRUCTURAL_PATTERN.matcher(line);
			if (!matcher.find()) {
				errorMessage("unknown structural pattern");
			}
			
			StructuralChangeInstance structuralChange = null;
			
			if (matcher.group(1)!=null) {//add attribute
				Attribute attribute = null;
				String attributeName = matcher.group(1);
				if (matcher.group(2) != null) { // NUMERIC
					attribute = new Attribute(attributeName, m_Data.numAttributes());
				} else if (matcher.group(3) != null) { // NOMINAL
					ArrayList<String> attributeValues = parseValues(matcher.group(3));
					attribute = new Attribute(attributeName,
							attributeValues,  m_Data.numAttributes());

				} else if (matcher.group(4) != null) { // STRING
					attribute = new Attribute(attributeName,
							(ArrayList<String>) null,  m_Data.numAttributes());
				} else if (matcher.group(5) != null) { // DATE
					String format = null;
					if (matcher.group(6) != null) {
						format = matcher.group(6);
					} else if (matcher.group(7) != null) {
						format = matcher.group(7);
					} else if (matcher.group(8) != null) {
						format = matcher.group(8);
					}
					attribute = new Attribute(attributeName, format,
							 m_Data.numAttributes());
				}
			
				structuralChange = new StructuralChangeInstance(StructuralChangeType.ADD_ATTRIBUTE, attribute);
				//System.out.println("ADD " + attribute);
				m_Data = StructuralChangeHelper.addAttribute(m_Data, attribute);
			} else if (matcher.group(9)!= null) {  // del attribute
				String attributeName = matcher.group(9);
				Attribute attribute = m_Data.attribute(attributeName);
				structuralChange = new StructuralChangeInstance(StructuralChangeType.DELETE_ATTRIBUTE, attribute);
				//System.out.println("DEL " + attributeName);
				m_Data = StructuralChangeHelper.delAttribute(m_Data, attributeName);
			} else if (matcher.group(10)!=null) { // add attribute value
				String attributeName = matcher.group(10);
				String val;
				if (matcher.group(11) != null) {
					val = matcher.group(11);
				} else if (matcher.group(12) != null) {
					val = matcher.group(12);
				} else if (matcher.group(13) != null) {
					val = matcher.group(13);
				} else {
					val = null;
					errorMessage("nominal value expected");
				}

				structuralChange = new StructuralChangeInstance(StructuralChangeType.ADD_ATTRIBUTE_VALUE, m_Data.attribute(attributeName), val);
				//System.out.println("ADD VALUE " + m_Data.attribute(attributeName) + " " + val);
				m_Data = StructuralChangeHelper.addAttributeValue(m_Data, attributeName, val);
			} else if (matcher.group(14)!=null) { // delete attribute value
				String attributeName = matcher.group(14);
				String val;
				if (matcher.group(15) != null) {
					val = matcher.group(15);
				} else if (matcher.group(16) != null) {
					val = matcher.group(16);
				} else if (matcher.group(17) != null) {
					val = matcher.group(17);
				} else {
					val = null;
					errorMessage("nominal value expected");
				}
				if (m_Data.attribute(attributeName).indexOfValue(val)<0) {
					errorMessage("value not found!");
				}
				structuralChange = new StructuralChangeInstance(StructuralChangeType.DELETE_ATTRIBUTE_VALUE, m_Data.attribute(attributeName), val);
				//System.out.println("DEL VALUE " + m_Data.attribute(attributeName) + " " + val);
				m_Data = StructuralChangeHelper.delAttributeValue(m_Data, attributeName, val);
			} else if (matcher.group(15) != null) { //add labels
				ArrayList<String> attributeValues = parseValues(matcher.group(15));
				ArrayList<String> labelValues = new ArrayList<String>(2);
				labelValues.add("0");
				labelValues.add("1");
				for (String value: attributeValues) {
					Attribute att = new Attribute(value, labelValues, m_Data.numAttributes());
					structuralChange = new StructuralChangeInstance(StructuralChangeType.ADD_LABEL, att);
					m_StructuralChangeList.add(structuralChange);
				}
				return; //TODO: refactorize
			} else if (matcher.group(16) != null) { //delete labels
				ArrayList<String> attributeValues = parseValues(matcher.group(16));
				for (String attributeName: attributeValues) {
					Attribute attribute = m_Data.attribute(attributeName);
					structuralChange = new StructuralChangeInstance(StructuralChangeType.DEL_LABEL, attribute);
					m_StructuralChangeList.add(structuralChange);
				}
				return; //TODO: refactorize
			}
			
			if (structuralChange == null) {
				errorMessage("Error parsing structural change");
			}
			m_StructuralChangeList.add(structuralChange);
		}

		private ArrayList<String> parseValues(String values) throws IOException {
			Matcher valueMatcher = ATTRIBUTE_DEF_VALUE_PATTERN.matcher("");							
			ArrayList<String> attributeValues = new ArrayList<String>();
			valueMatcher.reset(values);
			while (valueMatcher.find()) {
				String val;
				if (valueMatcher.group(1) != null) {
					val = valueMatcher.group(1);
				} else if (valueMatcher.group(2) != null) {
					val = valueMatcher.group(2);
				} else if (valueMatcher.group(3) != null) {
					val = valueMatcher.group(3);
				} else {
					val = null;
					errorMessage("nominal value expected");
				}
				attributeValues.add(val);
			}
			return attributeValues;
		}

		/**
		 * Reads a single instance using the tokenizer and returns it.
		 * 
		 * @param line
		 *            if method should test for carriage return after each
		 *            instance
		 * @return null if end of file has been reached
		 * @throws IOException
		 *             if the information is not read successfully
		 */
		protected Instance getInstanceSparse(String line) throws IOException {
			double[] m_ValueBuffer = new double[m_Data.numAttributes()];
			int[] m_IndicesBuffer = new int[m_Data.numAttributes()];
			int valIndex, numValues = 0, maxIndex = -1;
			double weight = 1.0;

			Matcher matcher = ATTRIBUTE_SPARSE_VALUE_PATTERN.matcher(line);
			boolean found = false;
			int start = 1;
			while (!found && matcher.find(start)) {
				String index = matcher.group(1);
				// Is index valid?
				try {
					m_IndicesBuffer[numValues] = Integer.valueOf(index)
							.intValue();
				} catch (NumberFormatException e) {
					errorMessage("index number expected");
				}
				if (m_IndicesBuffer[numValues] <= maxIndex) {
					errorMessage("indices have to be ordered");
				}
				if ((m_IndicesBuffer[numValues] < 0)
						|| (m_IndicesBuffer[numValues] >= m_Data
								.numAttributes())) {
					errorMessage("index out of bounds");
				}
				maxIndex = m_IndicesBuffer[numValues];

				// last att-value
				if (matcher.group(7) != null) {
					found = true;
				}

				String val = null;
				if (matcher.group(2) != null) { // missing
					// Check if value is missing.
					m_ValueBuffer[numValues] = Utils.missingValue();
					numValues++;
					start = matcher.end();
					continue;
				} else if (matcher.group(3) != null) {
					val = matcher.group(3);
				} else if (matcher.group(4) != null) {
					val = matcher.group(4);
				} else if (matcher.group(5) != null) {
					val = matcher.group(5);
				} else if (matcher.group(6) != null) {
					val = matcher.group(6);
				}

				switch (m_Data.attribute(m_IndicesBuffer[numValues]).type()) {
				case Attribute.NOMINAL:
					// Check if value appears in header.
				 	valIndex = m_Data.attribute(m_IndicesBuffer[numValues])
							.indexOfValue(val);
					if (valIndex == -1) {
						errorMessage("nominal value not declared in header");
					}
					m_ValueBuffer[numValues] = (double) valIndex;
					break;
				case Attribute.NUMERIC:
					// Check if value is really a number.
					try {
						m_ValueBuffer[numValues] = Double.valueOf(val)
								.doubleValue();
					} catch (NumberFormatException e) {
						errorMessage("number expected");
					}
					break;
				case Attribute.STRING:
					m_ValueBuffer[numValues] = m_Data.attribute(
							m_IndicesBuffer[numValues]).addStringValue(val);
					break;
				case Attribute.DATE:
					try {
						m_ValueBuffer[numValues] = m_Data.attribute(
								m_IndicesBuffer[numValues]).parseDate(val);
					} catch (ParseException e) {
						errorMessage("unparseable date: " + val);
					}
					break;
				default:
					errorMessage("unknown attribute type in column "
							+ m_IndicesBuffer[numValues]);
				}
				numValues++;
				start = matcher.end();
			}

			if (!found) {
				errorMessage("} not found");
			}

			matcher = WEIGHT_PATTERN.matcher(line);
			if (!matcher.find(start)) {
				errorMessage("Problem reading instance weight");
			}
			if (matcher.group(1) != null) {
				try {
					weight = Double.parseDouble(matcher.group(1));
				} catch (NumberFormatException e) {
					errorMessage("Problem reading instance weight");
				}
			}

			// Add instance to dataset
			double[] tempValues = new double[numValues];
			int[] tempIndices = new int[numValues];
			System.arraycopy(m_ValueBuffer, 0, tempValues, 0, numValues);
			System.arraycopy(m_IndicesBuffer, 0, tempIndices, 0, numValues);
			Instance inst = new SparseInstance(weight, tempValues, tempIndices,
					m_Data.numAttributes());
			inst.setDataset(m_Data);

			return inst;
		}

		/**
		 * Reads a single instance using the tokenizer and returns it.
		 * 
		 * @param line
		 *            if method should test for carriage return after each
		 *            instance
		 * @return null if end of file has been reached
		 * @throws IOException
		 *             if the information is not read successfully
		 */
		protected Instance getInstanceFull(String line) throws IOException {
			double[] instance = new double[m_Data.numAttributes()];
			double weight = 1.0;



			Matcher matcher = ATTRIBUTE_VALUE_PATTERN.matcher(line);
			int start = 0;
			for (int i = 0; i < m_Data.numAttributes(); i++) {
				if (!matcher.find(start)) {
					errorMessage("Error in attribute value " + i);
				}

				String val = null;
				if (matcher.group(1) != null) { // missing
					instance[i] = Utils.missingValue();
					start = matcher.end();
					continue;
				} else if (matcher.group(2) != null) {
					val = matcher.group(2);
				} else if (matcher.group(3) != null) {
					val = matcher.group(3);
				} else if (matcher.group(4) != null) {
					val = matcher.group(4);
				} else if (matcher.group(5) != null) {
					val = matcher.group(5);
				}

				switch (m_Data.attribute(i).type()) {
				case Attribute.NOMINAL:
					// Check if value appears in header.
					int index = m_Data.attribute(i).indexOfValue(val);
					if (index == -1) {
						errorMessage("nominal value not declared in header");
					}
					instance[i] = (double) index;
					break;
				case Attribute.NUMERIC:
					// Check if value is really a number.
					try {
						instance[i] = Double.valueOf(val).doubleValue();
					} catch (NumberFormatException e) {
						errorMessage("number expected");
					}
					break;
				case Attribute.STRING:
					instance[i] = m_Data.attribute(i).addStringValue(val);
					break;
				case Attribute.DATE:
					try {
						instance[i] = m_Data.attribute(i).parseDate(val);
					} catch (ParseException e) {
						errorMessage("unparseable date: " + val);
					}
					break;
				default:
					errorMessage("unknown attribute type in column " + i);
				}
				start = matcher.end();
			}


			matcher = WEIGHT_PATTERN.matcher(line);
			if (!matcher.find(start)) {
				errorMessage("Problem reading instance weight");
			}
			if (matcher.group(1) != null) {
				try {
					weight = Double.parseDouble(matcher.group(1));
				} catch (NumberFormatException e) {
					errorMessage("Problem reading instance weight");
				}
			}

			// Add instance to dataset
			Instance inst = new DenseInstance(weight, instance);
			inst.setDataset(m_Data);

			return inst;
		}

		/**
		 * Reads and stores header of an ARFF file.
		 * 
		 * @param capacity
		 *            the number of instances to reserve in the data structure
		 * @throws IOException
		 *             if the information is not read successfully
		 */
		protected void readHeader() throws IOException {
			String relationName = "";

			// read relation.
			boolean found = false;
			String line;
			Matcher matcher = RELATION_PATTERN.matcher("");
			while (!found && (line = reader.readLine()) != null) {
				matcher.reset(line);
				while (matcher.find()) {
					if (matcher.group(1) != null) { // @relation
						relationName = matcher.group(2);
						found = true;
					}
				}
			}
			if (!found) {
				errorMessage("keyword " + Instances.ARFF_RELATION + " expected");
			}

			// read attributes
			// Create vectors to hold information temporarily.
			ArrayList<Attribute> attributes = new ArrayList<Attribute>();

			found = false;
			matcher = ATTRIBUTE_PATTERN.matcher("");
			Matcher valueMatcher = ATTRIBUTE_DEF_VALUE_PATTERN.matcher("");
			while (!found && (line = reader.readLine()) != null) {
				matcher.reset(line);				
				if (!matcher.find()) {
					errorMessage("attribute unknown");
				}
				if (matcher.group(1) != null) {
					String attributeName = matcher.group(2);
					if (matcher.group(3) != null) { // NUMERIC
						attributes.add(new Attribute(attributeName, attributes
								.size()));
					} else if (matcher.group(4) != null) { // NOMINAL
						ArrayList<String> attributeValues = new ArrayList<String>();
						valueMatcher.reset(matcher.group(4));
						while (valueMatcher.find()) {
							String val;
							if (valueMatcher.group(1) != null) {
								val = valueMatcher.group(1);
							} else if (valueMatcher.group(2) != null) {
								val = valueMatcher.group(2);
							} else if (valueMatcher.group(3) != null) {
								val = valueMatcher.group(3);
							} else {
								val = null;
								errorMessage("nominal value expected");
							}
							attributeValues.add(val);
						}
						attributes.add(new Attribute(attributeName,
								attributeValues, attributes.size()));

					} else if (matcher.group(5) != null) { // STRING
						attributes.add(new Attribute(attributeName,
								(ArrayList<String>) null, attributes.size()));
					} else if (matcher.group(6) != null) { // DATE
						String format = null;
						if (matcher.group(7) != null) {
							format = matcher.group(7);
						} else if (matcher.group(8) != null) {
							format = matcher.group(8);
						} else if (matcher.group(9) != null) {
							format = matcher.group(9);
						}
						attributes.add(new Attribute(attributeName, format,
								attributes.size()));
					}
				} else if (matcher.group(10) != null) { // DATA found!
					found = true;
				} else if (matcher.group(11) != null) { // LABEL found!
					ArrayList<String> attributeValues = parseValues(matcher.group(15));
					ArrayList<String> labelValues = new ArrayList<String>(2);
					labelValues.add("0");
					labelValues.add("1");
					for (String value: attributeValues) {
						attributes.add(new Attribute(value,
								labelValues, attributes.size()));
					}				
				}
			}
			if (!found) {
				errorMessage("keyword " + Instances.ARFF_DATA + " expected");
			}

			// Check if any attributes have been declared.
			if (attributes.size() == 0) {
				errorMessage("no attributes declared");
			}

			m_Data = new Instances(relationName, attributes, 0);
		}

		/**
		 * Returns the header format
		 * 
		 * @return the header format
		 */
		public Instances getStructure() {
			return new Instances(m_Data, 0);
		}

		/**
		 * Returns the revision string.
		 * 
		 * @return the revision
		 */
		public String getRevision() {
			return RevisionUtils.extract("$Revision: 1 $");
		}

		public List<StructuralChangeInstance> getNextStructuralChanges() {
			return m_StructuralChangeList;
		}
	}

	/**
	 * Returns a string describing this Loader
	 * 
	 * @return a description of the Loader suitable for displaying in the
	 *         explorer/experimenter gui
	 */
	public String globalInfo() {
		return "Reads a source that is in arff (attribute relation file format) "
				+ "format. ";
	}

	/**
	 * Get the file extension used for arff files
	 * 
	 * @return the file extension
	 */
	public String getFileExtension() {
		return FILE_EXTENSION;
	}

	/**
	 * Gets all the file extensions used for this type of file
	 * 
	 * @return the file extensions
	 */
	public String[] getFileExtensions() {
		return new String[] { FILE_EXTENSION, FILE_EXTENSION_COMPRESSED };
	}

	/**
	 * Returns a description of the file type.
	 * 
	 * @return a short file description
	 */
	public String getFileDescription() {
		return "Dynamic Arff data files";
	}

	/**
	 * Resets the Loader ready to read a new data set or the same data set
	 * again.
	 * 
	 * @throws IOException
	 *             if something goes wrong
	 */
	public void reset() throws IOException {
		m_structure = null;
		setRetrieval(NONE);

		if (m_File != null && !(new File(m_File).isDirectory())) {
			setFile(new File(m_File));
		}
	}

	/**
	 * get the File specified as the source
	 * 
	 * @return the source file
	 */
	public File retrieveFile() {
		return new File(m_File);
	}

	/**
	 * sets the source File
	 * 
	 * @param file
	 *            the source file
	 * @throws IOException
	 *             if an error occurs
	 */
	public void setFile(File file) throws IOException {
		m_File = file.getPath();
		setSource(file);
	}

	/**
	 * Resets the Loader object and sets the source of the data set to be the
	 * supplied InputStream.
	 * 
	 * @param in
	 *            the source InputStream.
	 * @throws IOException
	 *             always thrown.
	 */
	public void setSource(InputStream in) throws IOException {
		m_File = (new File(System.getProperty("user.dir"))).getAbsolutePath();

		m_sourceReader = new BufferedReader(new InputStreamReader(in));
	}

	/**
	 * Determines and returns (if possible) the structure (internally the
	 * header) of the data set as an empty set of instances.
	 * 
	 * @return the structure of the data set as an empty set of Instances
	 * @throws IOException
	 *             if an error occurs
	 */
	public Instances getStructure() throws IOException {

		if (m_structure == null) {
			if (m_sourceReader == null) {
				throw new IOException("No source has been specified");
			}

			try {
				m_ArffReader = new DynamicArffReader(m_sourceReader);
				m_structure = m_ArffReader.getStructure();
			} catch (Exception ex) {
				throw new IOException(
						"Unable to determine structure as arff (Reason: "
								+ ex.toString() + ").");
			}
		}

		return new Instances(m_structure, 0);
	}

	/**
	 * Read the data set incrementally---get the next instance in the data set
	 * or returns null if there are no more instances to get. If the structure
	 * hasn't yet been determined by a call to getStructure then method should
	 * do so before returning the next instance in the data set.
	 * 
	 * @param structure
	 *            the dataset header information, will get updated in case of
	 *            string or relational attributes
	 * @return the next instance in the data set as an Instance object or null
	 *         if there are no more instances to be read
	 * @throws IOException
	 *             if there is an error during parsing
	 */
	public Instance getNextInstance(Instances structure) throws IOException {

		if (getRetrieval() == BATCH) {
			throw new IOException(
					"Cannot mix getting Instances in both incremental and batch modes");
		}
		setRetrieval(INCREMENTAL);

		Instance current = null;
		if (m_sourceReader != null) {
			current = m_ArffReader.readInstance();
			lastStructuralChangesList = m_ArffReader.getNextStructuralChanges();
		}

		if ((m_sourceReader != null) && (current == null)) {
			try {
				// close the stream
				m_sourceReader.close();
				m_sourceReader = null;
				// reset();
			} catch (Exception ex) {
				ex.printStackTrace();
			}
		}
		return current;
	}

	/**
	 * Returns the revision string.
	 * 
	 * @return the revision
	 */
	public String getRevision() {
		return RevisionUtils.extract("$Revision: 1 $");
	}

	/**
	 * Main method.
	 * 
	 * @param args
	 *            should contain the name of an input file.
	 */
	public static void main(String[] args) {
		runFileLoader(new DynamicArffLoader(), args);
	}

	@Override
	public Instances getDataSet() throws IOException {
		throw new IOException("Cannot use this method!");
	}

	public List<StructuralChangeInstance> getNextStructuralChanges() {
		return lastStructuralChangesList;
	}
}
